from bald_spider.http.request import Request
from bald_spider.spider import Spider
from tests.baidu_spider.items import BaiduSpiderItem


class BaiduSpider(Spider):
    start_urls = ['http://www.baidu.com', 'http://www.baidu.com']

    # @classmethod
    # def create_instance(cls, crawler):
    #     return cls()

    def parse(self, response):
        for i in range(5):
            url = "http://www.baidu.com"
            request = Request(url=url, callback=self.parse_page,dont_filter=True)
            yield request

    def parse_page(self, response):
        for i in range(5):
            url = "http://www.baidu.com"
            meta = {'dont_retry': False}
            request = Request(url=url, callback=self.parse_detail,meta=meta,dont_filter=False)
            yield request

    def parse_detail(self, response):
        #print(response.text)
        item = BaiduSpiderItem()
        item['url'] = response.url
        item['title'] = response.xpath('//title/text()').get()
        item['html'] = response.text
        yield item

    # async def spider_opened(self):
    #     print(f"我想在脚本开始的时候做一些事情...")
    #
    # async def spider_closed(self):
    #     print(f"我想在脚本结束的时候做一些事情...")

